package org.nerve.webmagic.task;

import org.apache.commons.lang3.StringUtils;
import org.nerve.webmagic.*;
import org.nerve.webmagic.bean.SpiderConfig;
import org.nerve.webmagic.downloader.JBrowserDriverDownloader;
import org.nerve.webmagic.downloader.PhantomJSDownloader;
import org.nerve.webmagic.pool.CounterPool;
import org.nerve.webmagic.utils.DirUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.utils.UrlUtils;

import java.util.TimerTask;

/**
 * 网页爬虫任务，用于执行爬虫任务
 * org.nerve.webmagic.task
 * Created by zengxm on 2016/4/21 0021.
 */
public class CollectTask extends TimerTask {
	protected static Logger logger = LoggerFactory.getLogger(CollectTask.class);
	protected static final int DOWNLOADER_SLEEPTIME=500;

	private final SpiderConfig config;

	public CollectTask(SpiderConfig config){
		this.config=config;

		if(StringUtils.isBlank(config.getDomain()))
			config.setDomain(UrlUtils.getDomain(config.getHomePage()).replace("www.",""));
		if(config.getLinkRule()==null)
			config.setLinkRule(".*"+ config.getDomain()+"/.*");
		if(StringUtils.isBlank(config.getScreenDirectory()))
			config.setScreenDirectory(DirUtil.PATH+"/screenShots/"+config.getDomain());
	}

	@Override
	public void run() {
		final CommonPageProcessor processor=new CommonPageProcessor(config);
		final Spider spider=Spider.create(processor)
				.addUrl(config.getHomePage())
				.thread(config.getThreadSize())
				.setScheduler(new DeepLimitScheduler(config.getDeep()))
				.clearPipeline()    //清空默认的consolePipeline
				.addPipeline(new CommonPipeline())
				.addPipeline(new JsonFilePipeline("/output/threadlocal"))
				;
		if(config.isPhantomjs()){
			spider.setDownloader(new PhantomJSDownloader(config));
		}else if(config.isJbrowser()){
			spider.setDownloader(new JBrowserDriverDownloader(config));
		}

		logger.debug("start CollectTask on Thread :"+Thread.currentThread().getName());
		//任务数加1
		CounterPool.addTask();
		spider.run();
		logger.debug("CollectTask end!");
		spider.close();
	}
}
